{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### TensorFlow数据引入",
   "id": "cf43ae298cda0e1e"
  },
  {
   "cell_type": "code",
   "id": "c80f8abef96c1ee",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:01:40.860904Z",
     "start_time": "2025-04-25T05:01:38.996344Z"
    }
   },
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.10.0'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### List列表数据",
   "id": "aaf1749dc1600390"
  },
  {
   "cell_type": "code",
   "id": "2b743f0e-5702-4a03-b140-b76c19fee10a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:01:42.596747Z",
     "start_time": "2025-04-25T05:01:42.548541Z"
    }
   },
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices([1,2,3])\n",
    "for element in dataset:\n",
    "  print(element)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(1, shape=(), dtype=int32)\n",
      "tf.Tensor(2, shape=(), dtype=int32)\n",
      "tf.Tensor(3, shape=(), dtype=int32)\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### Generator生成器",
   "id": "139ad6b77fc21c4b"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:04:07.367757Z",
     "start_time": "2025-04-25T05:04:07.364749Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import itertools\n",
    "def gen():\n",
    "    for i in itertools.count(1):\n",
    "        yield i, [1] * i"
   ],
   "id": "6a9aceb332d4f2a8",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:04:08.957036Z",
     "start_time": "2025-04-25T05:04:08.944700Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dataset = tf.data.Dataset.from_generator(\n",
    "    gen,\n",
    "    (tf.int64, tf.int64), #输入格式值\n",
    "    (tf.TensorShape(None), tf.TensorShape(None)), #生成的张量形状\n",
    ")"
   ],
   "id": "40b8fb24e554290a",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:04:10.616198Z",
     "start_time": "2025-04-25T05:04:10.594741Z"
    }
   },
   "cell_type": "code",
   "source": "list(dataset.take(5).as_numpy_iterator())",
   "id": "c44e8b7102a18526",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, array([1])),\n",
       " (2, array([1, 1])),\n",
       " (3, array([1, 1, 1])),\n",
       " (4, array([1, 1, 1, 1])),\n",
       " (5, array([1, 1, 1, 1, 1]))]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### 文本文件",
   "id": "d1297ebc15a8dbe2"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:10:24.835665Z",
     "start_time": "2025-04-25T05:10:24.832752Z"
    }
   },
   "cell_type": "code",
   "source": [
    "parent_dir = \"/Users/liyang/Work/IdeaProjects/AI-Learning/TensorFlow/files\"\n",
    "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']"
   ],
   "id": "60950e1a7838ef01",
   "outputs": [],
   "execution_count": 35
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:10:27.683314Z",
     "start_time": "2025-04-25T05:10:27.656327Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "\n",
    "def labeler(example, index):\n",
    "    return example, tf.cast(index, tf.int64)\n",
    "\n",
    "labeled_data_sets = []\n",
    "\n",
    "for i, file_name in enumerate(FILE_NAMES):\n",
    "    lines_dataset = tf.data.TextLineDataset(os.path.join(parent_dir, file_name))\n",
    "    labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n",
    "    labeled_data_sets.append(labeled_dataset)"
   ],
   "id": "c28ae00ea5e293cc",
   "outputs": [],
   "execution_count": 37
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:11:01.215903Z",
     "start_time": "2025-04-25T05:11:01.212338Z"
    }
   },
   "cell_type": "code",
   "source": [
    "BUFFER_SIZE = 50000\n",
    "BATCH_SIZE = 64\n",
    "TAKE_SIZE = 5000\n",
    "\n",
    "all_labeled_data = labeled_data_sets[0]\n",
    "for labeled_dataset in labeled_data_sets[1:]:\n",
    "    all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n",
    "\n",
    "all_labeled_data = all_labeled_data.shuffle(\n",
    "    BUFFER_SIZE, reshuffle_each_iteration=False)"
   ],
   "id": "fd73b713cc8db8fe",
   "outputs": [],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-04-25T05:11:11.819270Z",
     "start_time": "2025-04-25T05:11:11.389519Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for ex in all_labeled_data.take(5):\n",
    "    print(ex)"
   ],
   "id": "54769641a530979c",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'Should bear the destined trophy of his praise,'>, <tf.Tensor: shape=(), dtype=int64, numpy=0>)\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'Paris, Alcatho\\xc3\\xbcs, and Agenor led'>, <tf.Tensor: shape=(), dtype=int64, numpy=0>)\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'To whom Achilles matchless in the race.'>, <tf.Tensor: shape=(), dtype=int64, numpy=0>)\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b\"In Pleuron reign'd, and lofty Calydon:\">, <tf.Tensor: shape=(), dtype=int64, numpy=1>)\n",
      "(<tf.Tensor: shape=(), dtype=string, numpy=b'the blood of my fathers who were the noblest in Ephyra and in all'>, <tf.Tensor: shape=(), dtype=int64, numpy=2>)\n"
     ]
    }
   ],
   "execution_count": 44
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
